﻿using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.SharePoint.Administration;
using Microsoft.SharePoint;
using System.Net;
using HtmlAgilityPack;
using System.IO;
using System.Diagnostics;

namespace DeadLinkChecker
{
    public class DocumentLink
    {
        public DocumentLink(Uri link, HtmlNode linknode)
        {
            Link = link;
            LinkNode = linknode;
        }

        public Uri Link;
        public HtmlNode LinkNode;
    }

    public class LinkToCheck
    {
        public LinkToCheck(Uri link, Uri referer)
        {
            Link = link;
            Referer = referer;
        }

        public LinkToCheck(Uri link, Uri referer, HtmlNode linknode)
        {
            Link = link;
            Referer = referer;
            LinkNode = linknode;
        }

        public Uri Link;
        public Uri Referer;
        public Uri[] CrawlPath;
        public HtmlNode LinkNode;
    }

    public class DeadLinkCheckerJobDefinition : SPJobDefinition
    {
        public DeadLinkCheckerJobDefinition()
            : base()
        {
        }

        public DeadLinkCheckerJobDefinition(string jobName, SPService service, SPServer server, SPJobLockType targetType)
            : base(jobName, service, server, targetType)
        {
        }

        public DeadLinkCheckerJobDefinition(string jobName, SPWebApplication webApplication)
            : base(jobName, webApplication, null, SPJobLockType.ContentDatabase)
        {
            this.Title = "Dead Link Checker";
        }

        private DocumentLink[] GetReferencesFromDocument(HtmlDocument document, WebResponse response)
        {
            List<DocumentLink> links = new List<DocumentLink>();

            Queue<HtmlNode> nodes = new Queue<HtmlNode>();
            nodes.Enqueue(document.DocumentNode);

            while (nodes.Count > 0)
            {
                HtmlNode node = nodes.Dequeue();
                foreach (HtmlNode childnode in node.ChildNodes)
                {
                    nodes.Enqueue(childnode);
                }


                HtmlAttribute att = node.Attributes["href"];
                if (att != null)
                {
                    Uri u = null;
                    try
                    {
                        u = new Uri(att.Value);
                    }
                    catch
                    {
                        try
                        {
                            u = new Uri(response.ResponseUri, att.Value);
                        }
                        catch
                        {
                        }
                    }
                    if (u != null && (string.Compare(u.Scheme, "http", true) != 0 && string.Compare(u.Scheme, "https", true) != 0))
                    {
                        u = null;
                    }

                    if (u != null)
                    {
                        links.Add(new DocumentLink(u, node));
                    }
                }

                att = node.Attributes["src"];
                if (att != null)
                {
                    Uri u = null;
                    try
                    {
                        u = new Uri(att.Value);
                    }
                    catch
                    {
                        try
                        {
                            u = new Uri(response.ResponseUri, att.Value);
                        }
                        catch
                        {
                        }
                    }
                    if (u != null && (string.Compare(u.Scheme, "http", true) != 0 && string.Compare(u.Scheme, "https", true) != 0))
                    {
                        u = null;
                    }

                    if (u != null)
                    {
                        links.Add(new DocumentLink(u, node));
                    }
                }
            }
            return links.ToArray();
        }

        public HttpWebResponse GetResponse(string url, string referer, out string problem)
        {
            problem = null;

            SPSecurity.RunWithElevatedPrivileges(delegate
            {
                EventLog.WriteEntry(this.GetType().ToString(), "Requesting: " + url + " Referer:" + referer, EventLogEntryType.Information);
            });

            HttpWebResponse response = null;

            try
            {
                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
                request.Proxy = HttpWebRequest.GetSystemWebProxy();
                request.Referer = referer;
                request.Credentials = new CredentialCache();
                request.ImpersonationLevel = System.Security.Principal.TokenImpersonationLevel.Anonymous;
                request.PreAuthenticate = false;
                request.AuthenticationLevel = System.Net.Security.AuthenticationLevel.None;
                request.UseDefaultCredentials = false;
                request.Method = "GET";
                request.Timeout = 15000;
                request.AllowAutoRedirect = true;
                request.CookieContainer = null;
                request.MaximumAutomaticRedirections = 10;
                request.UnsafeAuthenticatedConnectionSharing = false;

                response = (HttpWebResponse)request.GetResponse();
            }
            catch (Exception ex)
            {
                problem = ex.Message;
                SPSecurity.RunWithElevatedPrivileges(delegate
                {
                    EventLog.WriteEntry(this.GetType().ToString(), "No response for: " + url + "\r\n" + ex.ToString(), EventLogEntryType.Warning);
                });
            }
            return response;
        }

        public DocumentLink[] GetReferences(HttpWebResponse response)
        {
            DocumentLink[] links = null;
            Stream responsestream = response.GetResponseStream();
            try
            {
                StreamReader sr = new StreamReader(responsestream);
                HtmlDocument document = new HtmlDocument();
                document.Load(responsestream);
                links = GetReferencesFromDocument(document, response);
            }
            catch (Exception ex)
            {
                SPSecurity.RunWithElevatedPrivileges(delegate
                {
                    //EventLog.WriteEntry(this.GetType().ToString(), "Can't get references for: " + response.ResponseUri.ToString() + "\r\n" + ex.ToString(), EventLogEntryType.Error);
                });
            }
            return links;
        }

        private bool IsAlreadyProcessed(List<string> successfullurls, string link)
        {
            return (!successfullurls.Contains(link.ToLower()));
        }

        private void MarkAsAlreadyProcessed(List<string> successfullurls, string link)
        {
            if (!IsAlreadyProcessed(successfullurls, link))
            {
                successfullurls.Add(link.ToLower());
            }
        }

        protected bool UrlInSiteCollection(SPSite site, string url)
        {
            bool samesite = false;
            try
            {
                using (SPSite urlsite = new SPSite(url))
                {
                    if (urlsite.ID.Equals(site.ID))
                    {
                        samesite = true;
                    }
                }
            }
            catch
            { }
            return samesite;
        }

        protected void DeleteListitems(SPListItemCollection listitems)
        {
            for (int idx = listitems.Count - 1; idx >= 0; idx--)
            {
                listitems[idx].Delete();
            }
        }

        protected void ReportProblem(LinkToCheck linktocheck, SPList bllist, string problem)
        {
            SPQuery query = new SPQuery();
            query.Query = string.Format("<Where><And>" +
            "<Eq><FieldRef Name=\"Referer\" /><Value Type=\"Text\">{0}</Value></Eq>" +
            "<Eq><FieldRef Name=\"Title\" /><Value Type=\"Text\">{1}</Value></Eq>" +
            "</And></Where>", linktocheck.Referer, linktocheck.Link);
            SPListItemCollection dupes = bllist.GetItems(query);
            if (dupes.Count == 0)
            {
                SPListItem listitem = bllist.Items.Add();
                listitem["Title"] = linktocheck.Link.ToString();
                listitem["Referer"] = linktocheck.Referer.ToString();
                //listitem["Html"] = (linktocheck.LinkNode!=null)?linktocheck.LinkNode.ToString():null;
                listitem["Problem"] = problem;
                listitem.Update();
            }
            else
            {
                foreach (SPListItem listitem in dupes)
                {
                    listitem["Problem"] = problem;
                    listitem.Update();
                }
            }
        }

        protected void DeleteFixedBrokenLinks(SPListItemCollection refererlinks, DocumentLink[] links)
        {
            for (int rlidx = refererlinks.Count - 1; rlidx >= 0; rlidx--)
            {
                SPListItem refererlink = refererlinks[rlidx];
                string brokenlink = (string)refererlink["Title"];
                bool found = false;

                if (links != null)
                {
                    foreach (DocumentLink link in links)
                    {
                        if (string.Compare(link.Link.ToString(), brokenlink, true) == 0)
                        {
                            found = true;
                        }
                    }
                }

                if (!found)
                { // No
                    // Remove the brokenlink.
                    refererlink.Delete();
                }
            }

        }

        protected void RecheckBrokenLinks(SPSite site)
        {
            SPList bllist = null;
            try
            {
                bllist = site.RootWeb.GetList("/Lists/BrokenLinks");
                bllist.ParentWeb.AllowUnsafeUpdates = true;
            }
            catch
            {
                bllist = null;
            }

            if (bllist != null)
            {
                #region Create a list of unique referers
                List<string> referers = new List<string>();
                for (int itemidx = bllist.Items.Count - 1; itemidx >= 0; itemidx--)
                {
                    SPListItem item = bllist.Items[itemidx];
                    string referer = (string)item["Referer"];
                    if (!referers.Contains(referer))
                    {
                        referers.Add(referer);
                    }
                }
                #endregion

                #region Check per referer
                foreach (string referer in referers)
                {
                    // Retrieve all BrokenLinks for this referer
                    SPQuery refererlinksquery = new SPQuery();
                    refererlinksquery.Query = string.Format("<Where><Eq><FieldRef Name=\"Referer\" /><Value Type=\"Text\">{0}</Value></Eq></Where>", referer);
                    SPListItemCollection refererlinks = bllist.GetItems(refererlinksquery);

                    // Check wether the referer url is not part of the SiteCollection anymore
                    if (!UrlInSiteCollection(site, referer))
                    { // The url is not part of the SiteCollection anymore
                        DeleteListitems(refererlinks);
                    }
                    else
                    { // The url is still part of the SiteCollection
                        string problem;
                        HttpWebResponse response = null;
                        try
                        {
                            response = GetResponse(referer, string.Empty, out problem);

                            // Was the request sucessfull?
                            if (response == null || response.StatusCode != HttpStatusCode.OK)
                            { // No
                                DeleteListitems(refererlinks);
                            }
                            else
                            { // Request was successfull
                                DocumentLink[] links = GetReferences(response);
                                DeleteFixedBrokenLinks(refererlinks, links);
                            }
                        }
                        finally
                        {
                            if (response != null) response.Close();
                        }
                    }
                }
                #endregion

                //bllist.Update();

                #region Check the remaining list of BrokenLinks
                for (int itemidx = bllist.Items.Count - 1; itemidx >= 0; itemidx--)
                {
                    SPListItem item = bllist.Items[itemidx];
                    string referer = (string)item["Referer"];
                    string link = (string)item["Title"];

                    HttpWebResponse response = null;
                    try
                    {
                        string problem = null;
                        response = GetResponse(link, referer, out problem);

                        if (response != null)
                        {
                            if (response.StatusCode == HttpStatusCode.OK)
                            {
                                item.Delete();
                            }
                            else
                            {
                                item["Problem"] = response.StatusDescription;
                                item.Update();
                            }
                        }
                        else
                        {
                            item["Problem"] = problem;
                            item.Update();
                        }
                    }
                    finally
                    {
                        if (response != null) response.Close();
                    }
                }
                #endregion
            }
        }

        protected void Crawl(LinkToCheck link, SPSite crawlsite, SPList bllist, List<string> successfullurls, Queue<LinkToCheck> linkstocheck)
        {
            HttpWebResponse response = null;
            try
            {
                string problem;
                response = GetResponse(link.Link.ToString(), link.Referer.ToString(), out problem);
                if (response == null)
                {
                    ReportProblem(link, bllist, problem);
                }
                else
                {
                    if (response.StatusCode == HttpStatusCode.OK)
                    {
                        string l1 = link.Link.ToString().ToLower();

                        if (!successfullurls.Contains(l1))
                        {
                            successfullurls.Add(l1);
                        }

                        string responseurl = response.ResponseUri.ToString().ToLower();
                        if (!successfullurls.Contains(responseurl))
                        {
                            successfullurls.Add(responseurl);
                        }

                        if (response.ContentType.IndexOf("text/html", StringComparison.InvariantCultureIgnoreCase) != -1 &&
                            UrlInSiteCollection(crawlsite, responseurl))
                        {
                            SPSecurity.RunWithElevatedPrivileges(delegate
                            {
                                EventLog.WriteEntry(this.GetType().ToString(), "Parsing: " + response.ResponseUri.ToString(), EventLogEntryType.Information);
                            }
                            );

                            Stream responsestream = null;
                            try
                            {
                                responsestream = response.GetResponseStream();
                                StreamReader sr = new StreamReader(responsestream);
                                HtmlDocument document = new HtmlDocument();
                                document.Load(responsestream);
                                DocumentLink[] documentlinks = GetReferencesFromDocument(document, response);
                                foreach (DocumentLink documentlink in documentlinks)
                                {
                                    string url = documentlink.Link.ToString().ToLower();

                                    if (successfullurls.Contains(url))
                                    {
                                        url = null;
                                    }

                                    foreach (LinkToCheck ltc in linkstocheck)
                                    {
                                        if (ltc.Link.Equals(documentlink.Link) && ltc.Referer.Equals(link.Link))
                                        {
                                            url = null;
                                        }
                                    }

                                    if (url != null)
                                    {
                                        //processedurls.Add(url);
                                        linkstocheck.Enqueue(new LinkToCheck(documentlink.Link, link.Link));
                                    }
                                }
                            }
                            finally
                            {
                                if (responsestream != null) responsestream.Close();
                            }
                        }
                    }
                    else
                    {
                        ReportProblem(link, bllist, problem);

                        SPSecurity.RunWithElevatedPrivileges(delegate
                        {
                            EventLog.WriteEntry(this.GetType().ToString(), response.ResponseUri.ToString() + " " + response.StatusCode.ToString(), EventLogEntryType.Warning);
                        }
                        );
                    }
                }
            }
            catch (Exception ex)
            {
                SPSecurity.RunWithElevatedPrivileges(delegate
                {
                    EventLog.WriteEntry(this.GetType().ToString(), link.ToString() + "\r\n" + ex.ToString(), EventLogEntryType.Error);
                }
                );
            }
            finally
            {
                if (response != null) response.Close();
            }
        }

        protected void CheckSiteCollection(SPSite site)
        {
            List<string> successfullurls = new List<string>();

            RecheckBrokenLinks(site);

            Queue<LinkToCheck> linkstocheck = new Queue<LinkToCheck>();

            SPList bllist = site.RootWeb.GetList("/Lists/BrokenLinks");
            bllist.ParentWeb.AllowUnsafeUpdates = true;
            if (bllist != null)
            {

                foreach (SPAlternateUrl alternateUrl in site.WebApplication.AlternateUrls)
                {
                    Uri StartUri = alternateUrl.Uri;
                    LinkToCheck startltc = new LinkToCheck(StartUri, null);
                    linkstocheck.Enqueue(startltc);
                }

                while (linkstocheck.Count > 0)
                {
                    LinkToCheck ltc = linkstocheck.Dequeue();
                    Crawl(ltc, site, bllist, successfullurls, linkstocheck);
                }
                Uri siteuri = new Uri(site.Url);
                foreach (SPWeb web in site.AllWebs)
                {
                    foreach (SPList list in web.Lists)
                    {
                        foreach (SPListItem item in list.Items)
                        {
                            if (item.File != null)
                            {
                                Uri uri = new Uri(siteuri, item.File.ServerRelativeUrl);
                                LinkToCheck ltc = new LinkToCheck(uri, null);
                            }
                        }
                    }
                }
            }
            else
            {
                SPSecurity.RunWithElevatedPrivileges(delegate
                {
                    EventLog.WriteEntry(this.GetType().FullName, string.Format("SiteCollection \"{0}\" skipped due to absense of BrokenLinks-list in the rootweb", site.Url), EventLogEntryType.Warning);
                }
                );
            }
            bllist.ParentWeb.AllowUnsafeUpdates = false;
        }

        public override void Execute(Guid contentDbId)
        {
            SPWebApplication webApplication = this.Parent as SPWebApplication;

            for (int idx = webApplication.Sites.Count - 1; idx >= 0; idx--)
            {
                SPSite site = null;
                try
                {
                    site = webApplication.Sites[idx];
                    using (SPSite systemSite = new SPSite(site.ID, site.SystemAccount.UserToken))
                    {
                        CheckSiteCollection(systemSite);
                    }
                }
                catch (Exception ex)
                {
                    SPSecurity.RunWithElevatedPrivileges(delegate
                    {
                        EventLog.WriteEntry(this.GetType().FullName, string.Format("Error occured processing Site Collection \"{0}\":\r\n{1}", site.Url, ex.ToString()), EventLogEntryType.Error);
                    });
                }
            }
        }

    }
}
